{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <div align=\"center\"> A CNN in Python WITHOUT frameworks </div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import skimage.data\n",
    "import matplotlib.pyplot as plt\n",
    "from mlxtend.data import loadlocal_mnist\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 全局配置plot参数\n",
    "plt.rcParams['font.sans-serif'] = 'SimHei'\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "plt.rcParams['figure.figsize'] = (12.0, 12.0)\n",
    "plt.rcParams['text.usetex'] = True\n",
    "plt.rcParams['text.latex.preamble'] = [r'\\usepackage{amsmath}']\n",
    "plt.rcParams['animation.html'] = 'jshtml'\n",
    "plt.rcParams['axes.labelsize'] = 16\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## zero padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def zero_pad(X, pad):\n",
    "    \"\"\"\n",
    "    Argument:\n",
    "        X: numpy array, shape(num_of_images, size_of_height, size_of_width, num_of_channels)\n",
    "        pad: integer, amount of zero padding\n",
    "    Return:\n",
    "        X_pad: numpy_array, shape(num_of_images, size_of_height + 2*pad,\n",
    "                size_of_width + 2*pad, num_of_channels)\n",
    "    \"\"\"\n",
    "    return np.pad(X, ((0, 0), (pad, pad), (pad, pad), (0, 0)), 'constant', constant_values=(0,0)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7ff9c902c5c0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAesAAADrCAYAAABJhL4VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAD9lJREFUeJzt3c9uG0e2x/HfGVwYRGAjjDWzzQWoBHczgAOaQV6AWRsIZHtpwAsafgHPA2Qlv8EwfgFjtPQqoZ9gFK/uVkRmfS2FSyGbcxeqliiGf1pudtcR+/sBAjVZdOpQIvnr6i5Wm7sLAADE9ZfcBQAAgPUIawAAgiOsAQAIjrAGACA4whoAgOAIawAAgqsc1mZ2YGZDM3u1ov0w/RxV7QsAgDaqFNZm1pckd59ImhW3F4zM7ETStEpfAAC0VdWR9VNJs7Q9lTRc8pjH7r6fAh0AANzQf1X8911JZ3O395Y8pm9mktR399eLjenw+EiSPvvss4dff/11xZJ2w2+//Za7hFC++uqr3CWE8uuvv35097/lrmOVTqfj9+7dy10GEN7Hjx9LvZerhvVGRUCb2fdmNlwcYbv7WNJYkr755hv/+eef6y7pVnj+/HnuEkJ59+5d7hJCMbP/5K5hnXv37umHH37IXQYQ3ng8LvVe3hjWKyaGTYvz1JLup/u6kk4X/u2BJLn7UWrrlSkKAABc2RjWaeS7yltJg7TdkzSRJDPruvtMF+exi4ll+5L++emlAgDQTpUmmLn7B0kys6GkWXFb0vu59idphH0y1w4AAEqqfM562cjb3R+uawcAAOWxghkAAMER1gAABEdYAwAQHGENAEBwhDUAAMHVvoIZgN2QvoI504qlgwHUh5E1gI1KXmEPQE0IawBllLnCHoCaENYAyth4hT0zG5nZsZkdn5+fN1cZ0AKENYCtcPexuw/cfdDpdHKXA+wUwhpAGWuvsAegXoQ1gDLe6uoSt5dX2APQDMIawEZrrrAHoAF8zxpAKVxBD8iHkTUAAMER1gAABEdYAwAQHGENAEBwhDUAAMER1gAABEdYAwAQHGENAEBwhDUAAMER1gAABFd5uVEzO9DFFXn67v76pu0AAGC9SiNrM+tLkrtPJM2K22XbAQDAZlUPgz/VxahZkqaShjdsBwAAG1QN666ks7nbezdsBwAAG2SfYGZmIzM7NrPj09PT3OUAABBO1bCeSbqftruSFtN2U7vcfezuA3cf7O0x8AYAYFHVsH4rqZe2e5ImkmRm3XXtAACgvEph7e4fJMnMhpJmxW1J7ze0AwCAkip/z9rdx0vue7iuHQAAlJd9ghmA24O1EoA8CGsApaTTWT/lrgNoI8IaQClpJcKzjQ8EsHWENQAAwRHWALZifoGj8/Pz3OUAO4WwBrAV8wscdTqd3OUAO4WwBgAgOMIaQCnp2vSD9BNAgyovigKgHdz9SNJR7jqANmJkDQBAcIQ1AADBEdYAAARHWAMAEBxhDQBAcIQ1AADBEdYAAARHWAMAEBxhDQBAcIQ1AADBsdwoAHyiH3/8sfE+nz9/3nif7969a7zPFy9eNN5nZIysAQAIjrAGACA4whoAgOAIawAAgiOsAQAIrnJYm9mBmQ3N7NWK9sP0c1S1LwAA2qhSWJtZX5LcfSJpVtxeMDKzE0nTKn0BANBWVUfWTyXN0vZU0nDJYx67+34KdAAAcENVF0XpSjqbu7235DF9M5Okvru/rtgfAACtU/sKZkVAm9n3ZjZcHGGnc9kjSep0Onr27FndJd0Kjx49yl1CKG/evMldAgBkUzWsZ5Lup+2upNP5RjM7kCR3P0ptvcX/gbuPJY0l6fPPP/eK9QCoydwk0X13/0fWYoCWqXrO+q2uArgnaSJJZtZN902L+yTtSzqu2B+ADMxsKGmSdq576TaAhlQKa3f/IF2+kWfFbUnv59qfpBH2yVw7gNulp6sJpFMtOUoGoD6Vz1mnPe3F+x6uawdwuyy8j/u6OKp2zfz8k7t37zZUGdAOrGAGoLS0lsIvy46SufvY3QfuPuh0OhmqA3YXYQ3gJoZ8BRNoHmENoBQzG819FZMJZkCDCGsAG6VwPjSzEzP7PXc9QNvUvigKgNsvLWb0Re46gLZiZA0AQHCENQAAwRHWAAAER1gDABAcYQ0AQHCENQAAwRHWAAAER1gDABAcYQ0AQHCsYAYAn+jZs2eN9/no0aPG+3zz5k3jfeI6RtYAAARHWAMAEBxhDQBAcIQ1AADBEdYAAARHWAMAEBxhDQBAcIQ1AADBEdYAAAS3lbA2s/6atgMzG5rZq230BQBA21QOazMbSvppRVtfktx9Imm2LtQBxJZ2uodmdpi7FqBtKod1CuKzFc1PJc3S9lTSsGp/AJqXdrS/T+/3PjveQLPqvpBHV9eDfK/m/gDUwN0/SPqQbvbSbQANyT7BzMxGZnZsZsd//PFH7nIArJHmnrxY0Xb5Xj4/P2+4MmC31R3WM0n303ZX0uniA9x97O4Ddx/cuXOn5nIAVOHuryW9MLPukrbL93Kn08lQHbC7agnruTfyW0m9tN2TNKmjPwD1MrP589RTSaOc9QBts43Z4AeSBuln4b10eZ6rmDE+4zwXcGsNdf0o2TRjLUDrVJ5g5u5Hko4W7ns4tz2u2geA7MaSnhQ75el9D6Ahdc8GB7AD3H2mi8CWFnbOAdQv+2xwAACwHmENAEBwhDUAAMER1gAABEdYAwAQHGENAEBwhDUAAMER1gAABEdYAwAQHCuYAcAt8vLly8b7fPDgQeN9fvfdd433GRkjawAAgiOsAQAIjrAGACA4whoAgOAIawAAgiOsAQAIjrAGACA4whoAgOAIawAAgiOsAdyImb3KXQPQNoQ1gNLMbCjp29x1AG1DWAMAEBxhDaAUM+u7+yR3HUAbbSWszay/pu0w/Rxtoy8A2dxf12hmIzM7NrPj8/PzpmoCWqFyWKdzWD+tecjIzE4kTav2BSCPMqNqdx+7+8DdB51Op6nSgFaofD1rd5+Y2dmahzzm0Blw6/XMrDe33Xf3D1krAlqkcliX0DczSeq7++sG+gOwZe5+JF2ezupmLgdondonmLn76zSy3kuHzAHcUulQ9z6jaqBZtY6szexAutwrP5XUW/KYkaSRJHGe68rLly9zlxDKgwcPcpcAANnUMrI2s+Iw2VRScb56X9Lx4mPnJ6XcuXOnjnIAALjVtjEb/EDSoBhFJ+8lKR0qe5LaTjh0BgDAzW1jNviRpKOF+x7ObY+r9gEAQJuxghkAAMER1gAABEdYAwAQHGENAEBwhDUAAMER1gAABNfE2uAAsJO+/PLLxvscjbjacBsxsgYAIDjCGgCA4AhrAACCI6wBAAiOsAYAIDjCGgCA4AhrAACCI6wBAAiOsAYAIDjCGgCA4AhrAKWY2WH6yXqXQMMIawBljczsRNI0dyFA23AhDwBlPXb3Se4igDYirAGU1TczSeq7++vFxnR4fCRJd+/ebbg0YLdxGBxAKe7+Oo2s98xsuKR97O4Ddx90Op0MFQK7i7AGsJGZHZjZQbp5KqmXsx6gbQhrAGVMJRXnq/clHWesBWgdzlkD2MjdP5jZyMzOJJ24+4fcNQFtQlgDKMXdx7lrANqqcljPLZCw7+7/WNJ+IGmmFTNIAQDAepXOWacZoZO0x91bnCFqZn1JSjNIZ8VtAABQXtUJZj1JRUBP9ecZok91Maou2v/0dQ8AALBepcPgC+ew+pLeLjykK+ls7vZelf4AAGijrUwwS4e3f/mUGaLzqx6xkAIAAH+2re9ZD1dMHptJup+2u7pYTOGa+VWP7ty5s6VyAADYHZXD2sxGRVAXE8zMrJua3+rqPHZPV4sqAACAkrYxG/zQzE7M7Pe5pvfSxUIKc4+bsZACAAA3V3WC2UTSF0vufzi3zUIKAABUwNrgAAAER1gDABAcYQ0AQHCENQAAwRHWAAAER1gDABAcYQ0AQHCENQAAwRHWAAAER1gDABAcYQ0AQHBbuZ41gN2XrlvfkyR3P8pcDtAqjKwBlPUihXQvBTeAhjCyBrCRmR1IOpGk4vr1AJrDyBpAGd9K2jOzvpm9WvYAMxuZ2bGZHZ+fnzdcHrDbCGsAZZ26+wfpcqR9jbuP3X3g7oNOp9N8dcAOI6wBlHEiaZq2p7oYaQNoCGENoIyJ0kzw9PPfGWsBWoewBrCRu08lzYrD33x1C2gWs8EBlOLu49w1AG3FyBoAgOAIawAAgiOsAQAIjrAGACA4whoAgOAqh3VaYnBkZocr2g+Lx1XtCwCANqoU1mY2lDRJX+nopduLRmY2v/oRAAC4gaoj656kIqCnulrhaN5jd99390nFvgAAaKVKi6IsLJLQl/R2ycP6ZiZJfS6tBwDAzZm7V/+fXFyIfrgujNO5618WR9jpXHZxPvvvkv63ckHV/FXSx8w1SNSxiDqu+x93v5e7iFXM7P8k/ecT/3mU33HdeJ675VOf53+7+982PWhby40uDeqFdYRPteQweRqdj9Pjj919sKWaPkmEGqiDOsrUkbuGdcp8+KwS5XdcN57nbqn7eW5lNngR1MUEMzPrpuapLq7WI0n7kkJ/wAAAENE2ZoMfmtmJmf0+1/RektKF6p+kEfZJceF6AABQXtUJZhNJXyy5/+Hc9k2u1BPhqj4RapCoYxF1XBeljjrs8nObx/PcLbU+z61MMAMAAPVhuVFck2b2r2o7MLOhmb3KXAer4gG3RBOfF20QIqw3hUBdH84l+m0knHI9/yX9DCX9tKKtL12e+pitC9M660hqXxWvxDK6Tb02WrOcb5M7gzlt+pvukvRe/jZ3HXUys3567R7U2U/2sC4ZAlv/cN7Ub1PhlOv5L5NqOFvR/FTSLG1PdbVyXdN1SDWvirdpGd0GXxutWc63yZ3BnEr+TXG7vEhfT+7V+brNHtYqFwJ1fDhv6repcMr1/G+qq+sBuperEF2silfnCGzTMrpNvTbatJxvYzuDmZX5m+4EM+vvwOtyreKbTpLk7q/r/MbTthZFqaJMCNSxZOmmfpsKp1zP/9aa+17/92Y23PYHQolldBt5bbRsOd9IO4O1Kfk33RX3cxfQgG+lcqt4VhVhZL1R2mOZSNpr42GjIM9/pqs3X1cXK9I1buHc0NJV8bbYV18XS+RmXR9gXR1BXhu4oSivrbq0YVQ957T4O9Z53rqRkfWKyS/T4vyU1oRAmSVLP9Gm8GkqnHI9/1LMrOvuM12MAIql9Hq6Wpmu6Tqmujo/uy/pnzV2u2pPuekdl09ezvcWCbEz2KBaR2EB9MysN7fd39EdkxNdHRGa6mKkfVRHR42MrN19vOS/4sP+ra4+ZC5DoIElSzf1u7S9Brme/5+kD//Bwt7h/Gp0xQSZWZ1vvBJ11L4q3oZldJt6bbRpOd/Gfqe5Lfub7hp3P0o7kfd1sfO1qya6/rr9d10dhVgUJY28p5J6xTkdM/u1WAkttZ+l9q3tjZbs91p7HXI9fyyXPkD/pYvf+X1dTOKaNP3auEEdO/HaaOr9ltOqv2neqlBFU+/BEGENAABWuxUTzAAAaDPCGgCA4AhrAACCI6wBAAiOsAYAIDjCGgCA4P4fhoSoQcxQ79oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x864 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# test zero_pad\n",
    "TX = np.random.randn(4, 3, 3, 2)\n",
    "TX_pad = zero_pad(TX, 3-1)\n",
    "TX.shape, TX_pad.shape\n",
    "fig, axes = plt.subplots(1, 2, figsize=(8,12))\n",
    "axes[0].imshow(TX[0,:,:,0])\n",
    "axes[1].imshow(TX_pad[0,:,:,0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## convolute one slice matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_single_step(slice_matrix, W, b):\n",
    "    \"\"\"\n",
    "    Argument:\n",
    "        slice_matrix: numpy array, the slice matrix by moving filter(kernel) selected. (h, w, c)\n",
    "        W: numpy array, filter weights parameters containded in a window - matrix of shape(h, w, c)\n",
    "        b: filter bias parameters in a window\n",
    "    Return:\n",
    "        Z: result of convoluting by sliding the window (W, b) on a slice matrix slice_matrix.\n",
    "    \"\"\"\n",
    "    return np.sum(np.multiply(slice_matrix, W)) + b # element-wise multiply, then sum + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[-0.45100213]]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test conv_single_step\n",
    "slice_matrix = np.random.randn(4, 4, 3)\n",
    "TW = np.random.randn(4, 4, 3)\n",
    "Tb = np.random.randn(1, 1, 1)\n",
    "conv_single_step(slice_matrix, TW, Tb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## feed forward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_forward_propagation():\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# gunzip t*-ubytez\n",
    "dataset_dir = '/home/lidong/Datasets/ML/mnist' \n",
    "X, Y = loadlocal_mnist(\n",
    "        images_path = dataset_dir + '/train-images-idx3-ubyte', \n",
    "        labels_path = dataset_dir + '/train-labels-idx1-ubyte')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(numpy.ndarray, (60000, 784), numpy.ndarray, (784,), numpy.ndarray, (60000,))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(X), X.shape, type(X[0]), X[0].shape, type(Y), Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def target_to_onehot(y):\n",
    "    t = np.zeros((10, 1))\n",
    "    t[int(y)] = 1.0\n",
    "    return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = [(x.reshape(-1,), target_to_onehot(y)) for x, y in zip(X, Y)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[0., 0., 0.],\n",
       "        [0., 0., 0.],\n",
       "        [0., 0., 0.]],\n",
       "\n",
       "       [[0., 0., 0.],\n",
       "        [0., 0., 0.],\n",
       "        [0., 0., 0.]],\n",
       "\n",
       "       [[0., 0., 0.],\n",
       "        [0., 0., 0.],\n",
       "        [0., 0., 0.]]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# a = np.random.randn(3,3,3)\n",
    "a = np.zeros((3,3,3))\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = a + np.array([[1],[2],[3]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
